# -*- coding: utf-8 -*-
# 创建时间：2021/7/19 17:12
from bs4 import BeautifulSoup
import requests
import time, math, random, json
import pymysql
import re
from hhy import FileUtil,DateUtil,ToolUtil,HttpUtil
__author__ = 'LuckyHhy'

class NewHouse(object):
    def __init__(self, xiaoqu, price, total):
        self.xiaoqu = xiaoqu
        self.price = price
        self.total = total

    def text(self):
        return self.xiaoqu + "," + \
                self.price + "," + \
                self.total



def create_headers():
    headers = dict()
    headers["User-Agent"] = HttpUtil.AgentRandom()
    headers["Referer"] = "http://www.ke.com"
    return headers




def main():
    with open("newhouse.txt", "w", encoding='utf-8') as f:
        # 开始获得需要的板块数据
        total_page = 1
        loupan_list = list()
        page = 'http://bj.fang.ke.com/loupan/'
        #print(page)
        headers = create_headers()
        response = requests.get(page, timeout=10, headers=headers)
        html = response.content
        soup = BeautifulSoup(html, "lxml")

        # 获得总的页数
        try:
            page_box = soup.find_all('div', class_='page-box')[0]
            matches = re.search('.*data-total-count="(\d+)".*', str(page_box))
            total_page = int(math.ceil(int(matches.group(1)) / 10))
        except Exception as e:
            print(e)

        print("总页数：{}".format(total_page))
        # 从第一页开始,一直遍历到最后一页
        headers = create_headers()
        for i in range(1, total_page + 1):
            page = 'http://bj.fang.ke.com/loupan/pg{0}'.format(i)
            print(page)
            response = requests.get(page, timeout=10, headers=headers)
            html = response.content
            soup = BeautifulSoup(html, "lxml")

            # 获得有小区信息的panel
            house_elements = soup.find_all('li', class_="resblock-list")
            for house_elem in house_elements:
                price = house_elem.find('span', class_="number")
                desc = house_elem.find('span', class_="desc")
                total = house_elem.find('div', class_="second")
                loupan = house_elem.find('a', class_='name')

                # 继续清理数据
                try:
                    price = price.text.strip() + desc.text.strip()
                except Exception as e:
                    price = '0'

                loupan = loupan.text.replace("\n", "")

                try:
                    total = total.text.strip().replace(u'总价', '')
                    total = total.replace(u'/套起', '')
                except Exception as e:
                    total = '0'

                # 作为对象保存
                loupan = NewHouse(loupan, price, total)
                print(loupan.text())
                loupan_list.append(loupan)

        for loupan in loupan_list:
            f.write(loupan.text() + "\n")




if __name__ == '__main__':
    main()
